package org.NooLab.openNLP.sources;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.EOFException;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.TreeMap;
import java.util.Vector;

 
import org.NooLab.chord.IndexedItemsCallbackIntf;
import org.NooLab.chord.MultiDigester;
 
import org.NooLab.utilities.files.DFutils;
import org.NooLab.utilities.files.DirectoryContent;
import org.NooLab.utilities.logging.PrintLog;
import org.NooLab.utilities.strings.StringsUtil;

 
/**
 * 
 * 
 * 
 * 
 * 
 * 
 */
public class LextDictionaryStorage implements Serializable{

	private static final long serialVersionUID = -3347619908482258163L;
	// =================================
	
	// object references ..............

	// main variables / properties ....


	String storagePath="";
	String language ="" ;
	
	boolean saveOnChange = true;
	boolean codeMapNeedsSave = false;
	boolean wordFormMapNeedsSave = false;
	
	/**  will be filled as an integrated representation onSsve and onLoad */
	ArrayList<WordLextEntry> wordLextEntries = new ArrayList<WordLextEntry> (); 
	ArrayList<String> wordLextEntriesMainWords = new ArrayList<String> (); 
 
	ArrayList<String> avoidanceItems = new ArrayList<String>() ;
	
	// this encodes in both directions ,
	// if  key=code then the object is a list of words with many entries...
	// otherwise, the value = ArrayList<> just contains 1 item = the code
	Map<String,ArrayList<String>> lexMainWordCode = new TreeMap<String,ArrayList<String>>() ;
	
	Map<String,String> wordFormMap = new TreeMap<String,String>() ;
	
	// constants ......................

	String wordFormsMappingFile = "lextMap.dat" ; // holds lexWordLabels
	String wordLextFileExt =  ".wle";
	
	int storageType = 0; // 0=file, 1=db
	
	int printLevel = 4;
	
	// volatile variables .............

	/** <p>
	 * this is created/filled onSave and onLoad = in-memory buffering
	 * &lt;String,WordLextEntry&gt; means 
	 * &nbsp;&nbsp;&nbsp;- &lt;any word of dictionary entry, WordLextEntry-object&gt; , OR 
	 * &nbsp;&nbsp;&nbsp;- &lt;code of category, WordLextEntry-object&gt;</p>
	 * 
	 * the Integer value points to a position in WordLextEntry.get(i)... this way we do
	 * not store the whole object multiple times 
	 * 
	 */
	Map<String,Integer> lexCatEntry = new TreeMap<String,Integer>() ;

	/**
	 * &lt;String,String&gt; means &lt;any word of dictionary entry, main word&gt;
	 */
	Map<String,String> lexWordLabels = new TreeMap<String,String>() ;
	
	// we need a container holding both maps for organizing storage... it does NOT make any sense
	// to store them separately !!
	LextMapStorage mappingsStorageContainer; 
	
 
	
	// helper objects .................
	 
	transient DFutils fileutil = new DFutils();
	transient StringsUtil strgutil = new StringsUtil();
	transient PrintLog out; 
	
	 
	// . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
	public LextDictionaryStorage( PrintLog outprn){
		out = outprn;
		fileutil.setOut( out );
		
		
	}
	
	public void initData(){ // called by setStoragePath()

										out.print(4, "loading persistent pointers and code mappings...") ;
		loadAvoidanceItemslist() ;
		loadWordCodeMap() ;
		loadWordFormsMap() ;
										out.print(4, "finished.") ;

	}

	
	// . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .
	
	public String createLextStorageEntryFilename(String wordlabel){
		String filename="", str;
		
		wordlabel = wordlabel.replace(".", "-");
		
		if (wordlabel.length()==0){
			return filename;
		}
		 
		 
		if (strgutil.isCharUpperCase(wordlabel,0)==true){
			str = wordlabel.substring(0,1)+wordlabel;
			wordlabel = str;
		}
		// storagepath has been set by "class LanguageProcessorCore{}" and equals its variable "conjDataPath" ;
		filename = storagePath + wordlabel + getWordLextFileExt() ;

		if (filename.contains("/.")){
			filename="";
		}
		return filename;
	}
	
	public int saveSingleEntry( WordLextEntry wordLexTentry){
		
		int result=-1;
		String filename ,wordlabel ;
		FileOutputStream fileOut;
		BufferedOutputStream bout;
		ObjectOutputStream objout ;
		
		
		if (wordLexTentry==null){
			return result;
		}
												
		wordlabel = wordLexTentry.getMainWord() ;
		wordlabel = wordlabel.replace("+", " "); wordLexTentry.setMainWord(wordlabel) ;
		filename = createLextStorageEntryFilename( wordlabel );
		 									     out.print(4,"Writing LexT-object for <"+wordlabel+">... ");
		if (fileutil.fileexists(filename) ){
			// make it optional fileutil.manageBakFile(filename, 3);
		}
		
		try{
			wordLexTentry.lastAccessTimeStamp = System.currentTimeMillis() ;
			
												 out.print(4,"Writing LexT-object physically (1)... ");
		    fileOut = new FileOutputStream( filename );
		    bout = new BufferedOutputStream(fileOut);
            objout = new ObjectOutputStream(bout);

            									 out.print(4,"Writing LexT-object physically (2)... ");
            objout.writeObject(wordLexTentry);

            									 out.print(4,"closing all output streams... ");
            objout.close();
            bout.close();
            fileOut.close();
            result=0 ;
            									 out.print(4,"output streams closed. ");
		}catch(Exception e){
			result = -7;
			e.printStackTrace();
		}
		 
		if (result==0){
			// we need a mirror with String baseform !
			wordLextEntries.add(wordLexTentry) ;
			
			addWordCodeMapItem( wordLexTentry.mainWord, wordLexTentry.getCategoryCode()  );
			
			// insertEntryToMaps( wordlabel, wordLexTentry);
			// saveWordFormsMappingsFile() ; // save the map "lexWordLabels", which maps anyword -> code of mainword
		}else{
			out.print(1, "writing Lext-file for <"+wordlabel+"> failed;\n"+
					     "filename should have been : "+filename);
		}
												 out.print(4,"post-output finished. ");
		return result;
	}
	
	
	public WordLextEntry loadSingleEntry( String wordlabel ){
		

		 
		String filename ="", catlabelforfile;
		int p = -1;
		
		WordLextEntry wle = null;
		// Map<String, String> wlexMap ;
		
		
		catlabelforfile = wordlabel;
		if (strgutil.isCharUpperCase(wordlabel,0)==true){
			catlabelforfile = wordlabel.substring(0,1)+wordlabel;
		}
		
		// storagepath has been set by "class LanguageProcessorCore{}" and equals its variable "conjDataPath" ;
		filename = storagePath + catlabelforfile + getWordLextFileExt() ;
		if ( (fileutil.fileexists(filename)==false) ){
		 	return null ;
		}
		                       							out.print(4, "                              ... getWordLextEntryPosition() " );
		p = getWordLextEntryPosition( wordlabel ) ;
														out.print(4, "                              ...  " );
		if (p>=0){
			wle = wordLextEntries.get(p) ;
			// wle.initAfterObjLoad( verbFormsMap );
			// contains ?
			lexWordLabels.put( wordlabel, wle.mainWord );
			return wle;
		}
		
                                          if (out!=null)out.print(4,"    Loading word lex Object ("+wordlabel+")...");

		wle = loadSingleEntryByFilename(filename);
		
		// it is not in the collection, add it
													    out.print(4, "                              ... post admin ... " );
		if ((p<0) && (wle!=null)){
			if ( wle.getMainWord().length()>0){
														out.print(4, "                              ... post admin (1) " );
				wordLextEntries.add(wle) ;
														out.print(4, "                              ... post admin (2) " );
				insertEntryToMaps( wordlabel, wle);
														out.print(4, "                              ... post admin (3) " );
				// saveWordFormsMappingsFile() ; // save the map "lexWordLabels", which maps anyword -> code of mainword
			}
		}
														out.print(4, "                              ... post admin finished. " );
		
		return wle ;
		
	}

	public WordLextEntry loadSingleEntryByFilename( String filename ){
		WordLextEntry wle = null;
		// Map<String, String> wlexMap ;
		
		ObjectInputStream objistream ;
		BufferedInputStream bins;
		FileInputStream fileIn;

		try {
															out.print(5, "                              ... physical access ... ");
															
			fileIn = new FileInputStream(filename);
			bins = new BufferedInputStream(fileIn);
			objistream = new ObjectInputStream(bins);

			wle = (WordLextEntry) objistream.readObject();

		 
															out.print(5, "                          Closing all input streams...");
			objistream.close();
			bins.close();
			fileIn.close();
															out.print(5, "                              ... physical access closed. ");

		} catch (Exception e) {
			out.print(1, "\nproblems reading binary file in loadSingleEntryByFilename() : "+filename);
			e.printStackTrace();
		}
		
		if (wle!=null){
			// check, whether this entry is contained in our word<->Code map
			// addWordCodeMapItem( String mainword, ArrayList<String> codes){
		}
		return wle;
	}
	
	
	/**
	 * for multi-threaded load of files
	 */
	protected void loadSelectedFile( String srcfilename, int processID){
		
		// insertEntryToDedicatedMap(wordlabel, wle);
		// these maps are collected across the threads, then collated together
	}
	
	public void bootloadBufferedWordLextEntries(){
		
		String filename, wordlabel;
		WordLextEntry wle ;
		ArrayList<String> wleFiles;
		ArrayList<String> filfilter = new ArrayList<String> ();
		ArrayList<String> dirfilter = new ArrayList<String> ();
		
		DirectoryContent dir = new DirectoryContent() ;
		int i,stepw ,threadcount=1;
		
		FileDigester fileDigester = new FileDigester() ;
		
														   	out.print(2,"loading buffered WordLextEntries (lookup)...");
        i=0;
        dirfilter.add(".wle");
		// wleFiles = dir.completeListofFiles( storagePath, filfilter, dirfilter, false) ;
		wleFiles = dir.getFileList( "", ".wle", storagePath) ;
		
		// multi-threaded loading of files
		// fileDigester.digestingFiles(wleFiles, threadcount)

		
		stepw = Math.round(wleFiles.size()/20);
		i=0;
		while (stepw > 2000){
			i++;
			stepw = Math.round(wleFiles.size()/(20*(i*10)));
		}
															out.print(2,"loading buffered WordLextEntries (n="+wleFiles.size()+")...");
		for (i=0;i<wleFiles.size();i++){
															out.printprc(2, i, wleFiles.size(), stepw, "") ;
			filename = wleFiles.get(i) ;
			
			if ((filename.length()>1) && (filename.contains(".wle"))){
				try{
					if ( (filename.indexOf("/")==0) && (filename.indexOf("\\")==0)){
						filename = fileutil.createPath( storagePath, filename) ;
					}
					wle = loadSingleEntryByFilename(filename);

					wordlabel = wle.getMainWord();
					insertEntryToMaps(wordlabel, wle);
					
				} catch(Exception e){
					out.print(1, "file failed to load : " + filename);
					e.printStackTrace() ; 
				}
			}
		} // i-> all found files
		i=0;
															out.print(4,"bootloadBufferedWordLextEntries() finished.");
	}

	public void insertWordLabelToMaps( String mainwordlabel, String additionallabel){
		int p;
		WordLextEntry wle;
		/*
		if (lexCatEntry.containsKey( mainwordlabel ) ){
			p = lexCatEntry.get(mainwordlabel);
			
			if (p>=0){
				wle = wordLextEntries.get(p);
				insertEntryToMaps(additionallabel, wle);
			}
		}
		*/
	}

	public boolean isMarkedForAvoidance( String wordlabel) {
		boolean rb=false;
		int p;
		
		rb = avoidanceItems.contains(wordlabel) ;
		
		return rb;
	}
	
	/**
	 * we maintain a list in order to avoid slow down by trying to resolve the unresolvable repeatedly
	 * it is a serialized ArrayList<String> list, but we also create wle files which contain a respective flag
	 * 
	 * 
	 * @param wordlabel
	 */
	public void markForAvoidance(String wordlabel){
		String filename;
		
		WordLextEntry wordLexTentry = new WordLextEntry() ;
		

		wordLexTentry.mainWord = wordlabel;
		wordLexTentry.emptyflag = 1;
		 	
		
		filename = createLextStorageEntryFilename( wordlabel );
	     
		if (fileutil.fileexists(filename)==false ){
			saveSingleEntry( wordLexTentry );
		}
		// this prevents the wle object remaining empty... so, no remote access will take place
	 	
		if (avoidanceItems.indexOf(wordlabel)<0){
			avoidanceItems.add(wordlabel);
			saveAvoidanceItemslist();
		}
		
	}
	
	
	public void addWordFormMapping( String keyStr, String valueStr){
		
		if (keyStr.length()==0){
			return;
		}
		if (valueStr.length()==0){
			return;
		}
		 
		if (wordFormMap.containsKey(keyStr)==false){
		
			wordFormMap.put( keyStr, valueStr) ;
			wordFormMapNeedsSave = true;
		}
		
	}

	@SuppressWarnings("unchecked")
	public void loadWordFormsMap(){
		// 

		String filename  ;
		
		ObjectInputStream objistream ;
		BufferedInputStream bins;
		FileInputStream fileIn;
		
		
		filename =  wordFormsMappingFile;
		filename = filename.replace("Map.","WFormMap.") ;
		filename = fileutil.createPath( storagePath ,filename);
		
		if ( (fileutil.fileexists(filename)==false) ){
		 	return   ;
		}
		               

		try {
															
			fileIn = new FileInputStream(filename);
			bins = new BufferedInputStream(fileIn);
			objistream = new ObjectInputStream(bins);

			wordFormMap = ( Map<String,String>) objistream.readObject();

			objistream.close();
			bins.close();
			fileIn.close();
															
			wordFormMapNeedsSave = false;
			
		} catch (Exception e) {
			e.printStackTrace();
		}

	}

	public void saveWordFormsMap(){
		// 

		String filename  ;
		FileOutputStream fileOut;
		BufferedOutputStream bout;
		ObjectOutputStream objout ;
		
		
		// wordFormsMappingFile = "lextMap.dat" ;
		filename =  wordFormsMappingFile;
		filename = filename.replace("Map.","WFormMap.") ;
		filename = fileutil.createPath( storagePath ,filename);
		 
		
		try{
			 
		    fileOut = new FileOutputStream( filename );
		    bout = new BufferedOutputStream(fileOut);
            objout = new ObjectOutputStream(bout);

            // it's a simple ArrayList<String>, so far...							 
            objout.writeObject( wordFormMap );

            									 
            objout.close();
            bout.close();
            fileOut.close();
           
            									 
		}catch(Exception e){
			 
			e.printStackTrace();
		}
		
	}
	
	
	public String getWordFormMapping( String rqlabel ){
		String rStr = "";
		
		if (rqlabel.length()==0){
			return rStr;
		}
		if (wordFormMap.containsKey(rqlabel)==true){
			
			rStr = wordFormMap.get( rqlabel) ;
			// rStr now contains the baseform which matches the requested  "rqlabel"
			if (rStr==null){
				rStr="" ;
			}
			
		}
		return rStr;
	}
	 
	 
	/**
	 * 
	 * the lists "codes" and "catlabels" run in parallel (like a ladder), i.e. a particular index 
	 * links entries in the two lists
	 * 
	 * @param mainword
	 * @param codes
	 * @param catlabels
	 */
	public void addWordCodeMapItem( String mainword, ArrayList<String> codes){ // , ArrayList<String> catlabels
		int i;
															out.print(4, "adding main word + code to map (n="+codes.size()+") for  "+mainword); 
		i=0;
		for (i=0;i<codes.size();i++){
			addWordCodeMapItem(mainword, codes.get(i)) ;
			// we store ONLY the "mainword", since the rest (incl. words from embedded sub-cat ) 
			// can be retrieved from the wle file
			// DO NOT DO THIS: addWordCodeMapItem(catlabels.get(i),codes.get(i) ) ;
		}
		i=0;

															out.print(4, "adding to map finished.  "); 
	}
	
	public void addWordCodeMapItem( String mainword, String code){
	
		boolean wordContained, codeContained ;
		ArrayList<String> mapli ;
		ArrayList<String> mappedList = new ArrayList<String>();
		
		if (code.length()==0){
			return;
		}
		if (mainword.length()<=1){
			return;
		}
		
		
		wordContained = lexMainWordCode.containsKey(mainword) ;
		codeContained = lexMainWordCode.containsKey(code) ;
		
	 
		if (wordContained==false){
			mappedList.add( code );
			lexMainWordCode.put(mainword, mappedList);
		} else {
			mappedList = lexMainWordCode.get(mainword) ;
			if (mappedList.contains(code)==false){
				mappedList.add(code);
			}
		}
		
		mappedList = new ArrayList<String>();
		if (codeContained==false){
			mappedList.add( mainword );
			lexMainWordCode.put(code, mappedList);
		} else{
			mappedList = lexMainWordCode.get(code) ;
			if (mappedList.indexOf(mainword)<0){
				mappedList.add(mainword);
			}
		}

		saveWordCodeMap();
		codeMapNeedsSave = true;
	
	
	}
	
	public ArrayList<String> getMainWordFromCode( String catcode){
		
		ArrayList<String> mainwords  ;
		//Map<String,ArrayList<String>> lexMainWordCode 
	
		mainwords = lexMainWordCode.get(catcode) ;
		
		if (mainwords==null){
			mainwords = new ArrayList<String>() ;
		}
		return mainwords;
	}
	
	/*
	 * 
	 * if(mappedList == null) {
				it.remove();
			} else {
				entry.setValue( mappedList );
			}
			
	 */
	
	
	
	public Map<String,String> exportAllCodes(){
		
		Map<String,String> codelist = new TreeMap<String,String>() ;
		ArrayList<String> mappedList ;
		Map.Entry<String, ArrayList<String>> entry ;
		
		String str, valuestr;
		
		for (Iterator<Map.Entry<String, ArrayList<String>>> it = lexMainWordCode.entrySet().iterator(); it.hasNext();) {
			
			entry = it.next();
			
			str = entry.getKey() ;
			
			if (strgutil.isNumericX(str)){
				mappedList = entry.getValue() ;
				valuestr = strgutil.collapse(mappedList, ",") ;
				codelist.put(str, valuestr) ;
			}
		} // -> the whole map
		
		return codelist;
	}
	
	@SuppressWarnings("unchecked")
	private void loadWordCodeMap(){
		

		String filename , str="" ;
		int err=1;
		
		ObjectInputStream objistream ;
		BufferedInputStream bins;
		FileInputStream fileIn;
		
		
		filename =  wordFormsMappingFile;
		filename = filename.replace("Map.","CodeMap.") ;
		filename = fileutil.createPath( storagePath ,filename);
		
		if ( (fileutil.fileexists(filename)==false) ){
		 	return   ;
		}
		               

		try {
															
			fileIn = new FileInputStream(filename);
			bins = new BufferedInputStream(fileIn);
			objistream = new ObjectInputStream(bins);
			
			try{
				lexMainWordCode = (Map<String, ArrayList<String>>) objistream .readObject();
 
				codeMapNeedsSave = false;
				err = 0;
				
			}catch( EOFException eofx){
				str = eofx.getMessage() + "\n" + eofx.getCause() ;
				str = strgutil.collapse( eofx.getStackTrace(), "\n");
					 
			}
			
			objistream.close();
			bins.close();
			fileIn.close();
			
			if (err>0){
				saveWordCodeMap();
			}
		} catch (Exception e) {
			e.printStackTrace();
		}

	} 
	
	protected void saveWordCodeMap(){

		 
		String filename  ;
		FileOutputStream fileOut;
		BufferedOutputStream bout;
		ObjectOutputStream objout ;
		
		
		// wordFormsMappingFile = "lextMap.dat" ;
		filename =  wordFormsMappingFile;
		filename = filename.replace("Map.","CodeMap.") ;
		filename = fileutil.createPath( storagePath ,filename);
		 
		
		try{
			 
		    fileOut = new FileOutputStream( filename );
		    bout = new BufferedOutputStream(fileOut);
            objout = new ObjectOutputStream(bout);

            // it's a simple ArrayList<String>, so far...							 
            objout.writeObject( lexMainWordCode );

            									 
            objout.close();
            bout.close();
            fileOut.close();
           
            									 
		}catch(Exception e){
			 
			e.printStackTrace();
		}
		 
	}
	
	
	protected void saveAvoidanceItemslist(){
		
 
		String filename  ;
		FileOutputStream fileOut;
		BufferedOutputStream bout;
		ObjectOutputStream objout ;
		
		
		// wordFormsMappingFile = "lextMap.dat" ;
		filename =  wordFormsMappingFile;
		filename = filename.replace("p.","pavoid.") ;
		filename = fileutil.createPath( storagePath ,filename);
		 
		
		try{
			 
		    fileOut = new FileOutputStream( filename );
		    bout = new BufferedOutputStream(fileOut);
            objout = new ObjectOutputStream(bout);

            // it's a simple ArrayList<String>, so far...							 
            objout.writeObject(avoidanceItems);

            									 
            objout.close();
            bout.close();
            fileOut.close();
           
            									 
		}catch(Exception e){
			 
			e.printStackTrace();
		}
		 
		 
	}
	
	@SuppressWarnings("unchecked")
	private void loadAvoidanceItemslist(){
		
		String filename  ;
		
		ObjectInputStream objistream ;
		BufferedInputStream bins;
		FileInputStream fileIn;
		
		
		filename =  wordFormsMappingFile;
		filename = filename.replace("p.","pavoid.") ;
		filename = fileutil.createPath( storagePath ,filename);
		
		if ( (fileutil.fileexists(filename)==false) ){
		 	return   ;
		}
		                       							 
	
		
	
		try {
															
			fileIn = new FileInputStream(filename);
			bins = new BufferedInputStream(fileIn);
			objistream = new ObjectInputStream(bins);
	
			avoidanceItems = (ArrayList<String>) objistream.readObject();
	
			objistream.close();
			bins.close();
			fileIn.close();
															
	
		} catch (Exception e) {
			e.printStackTrace();
		}
	
	}

	private void insertEntryToMaps( String requestedLabel, WordLextEntry wle ){
		 
		String str ;
		ArrayList<String> catcodes, catlabels ;
		int pos;
		// Map<String,Integer>, the int i points to WordLextEntry.get(i) 
		
		/*
		pos = wordLextEntries.size()-1;
		
		catcodes  = wle.getCategoryCode() ;
		catlabels = wle.getCategoryLabel() ;
				
		lexCatEntry.put( wle.mainWord, pos );
		lexWordLabels.put( "p"+pos, wle.mainWord );
		
		lexWordLabels.put( requestedLabel, wle.mainWord );
		
		mappingStringsToPos(catcodes, pos);  // and...   
		mappingStringsToPos(catlabels, pos);
		
		for (int i=0;i<wle.getCategoryWordsList().size();i++){
			mappingStringsToPos( wle.getCategoryWords(i), pos);
		}

		mappingStringsToPos( wle.getEmbeddingShortPhrases(), pos);
		mappingStringsToPos( wle.getMainSimilars(), pos);
		mappingStringsToPos( wle.getMainSimilarsForonly(), pos);
		mappingStringsToPos( wle.getMetonymials(), pos);
		
		str = wle.getBaseform();
		
		if (str.length()>0){
			lexCatEntry.put( str, pos);
			lexWordLabels.put( str, wle.mainWord );
		}
		*/
	}

	private void mappingStringsToPos(ArrayList<String> strs, int pos){
		
		String str;
		
		for (int i=0;i<strs.size();i++){
			str = strs.get(i) ;
			if (str.length()>0){
				lexCatEntry.put( str, pos );
			}
		}// i->
		str="";
	}
	 
 
	
	public ArrayList<String> getAvoidanceItems() {
		return avoidanceItems;
	}

	public void setAvoidanceItems(ArrayList<String> avoidanceItems) {
		this.avoidanceItems = avoidanceItems;
	}

	public Map<String, ArrayList<String>> getLexMainWordCode() {
		return lexMainWordCode;
	}

	public void setLexMainWordCode(Map<String, ArrayList<String>> lexMainWordCode) {
		this.lexMainWordCode = lexMainWordCode;
	}

	protected int getWordLextEntryPosition( WordLextEntry wle ) {
		
		int resultPos = -1;
		String mword;
		
		mword = wle.getMainWord() ;
		if (lexCatEntry.containsKey(mword)){
			resultPos = lexCatEntry.get(mword);
		}
		
		return resultPos;
	}
	
	
	protected int getWordLextEntryPosition( String anyword ) {
		
		int resultPos = -1;
		WordLextEntry wle;
		int p1 ;
		String mword="";
		
		// contains?
		// get mainword
		
		if (lexWordLabels.containsKey(anyword)){
			mword = lexWordLabels.get(anyword);
		}
		
		if ( (mword!=null) && (mword.length()>1) && (lexCatEntry!=null) && (lexCatEntry.containsKey(mword)) ){
			p1 = lexCatEntry.get(mword);

			if (p1>=0){
				wle = wordLextEntries.get(p1);
				wle.getMainWord();
			}
			resultPos = p1;
		}
		return resultPos;
	}
	
	protected WordLextEntry getWordLextEntry( String anyword ) {
		 
		WordLextEntry wle = null;
		int p1 ;
		String mword="";
		boolean b1,b2 ;
		
		// contains?
		// get mainword
		
		
		b1 = lexWordLabels.containsKey(anyword);
		b2 = lexCatEntry.containsKey(anyword);
		
		if ((b1) && (b2==false)){
			mword = lexWordLabels.get(anyword);
		}else{
			mword=anyword;
		}
		
		if ( (mword!=null) && (mword.length()>1) && (lexCatEntry!=null) && (lexWordLabels.containsKey(mword)) ){
		 
		
			p1 = lexCatEntry.get(mword);

			wle = wordLextEntries.get(p1);
			wle.getMainWord();
		}
	 
		return wle;
	}
	
	
	// ------------------------------------------------------------------------
	
	public String getWordLextFileExt() {
		return wordLextFileExt;
	}



	public void setWordLextFileExt(String wordLextFileExt) {
		this.wordLextFileExt = wordLextFileExt;
	}



	public String getStoragePath() {
		return storagePath;
	}



	public void setStoragePath(String storagePath) {
		this.storagePath = storagePath;
		
		initData();
       														
	}



	public boolean isCodeMapNeedsSave() {
		return codeMapNeedsSave;
	}

	public boolean isWordFormMapNeedsSave() {
		return wordFormMapNeedsSave;
	}

	public String getLanguage() {
		return language;
	}



	public void setLanguage(String language) {
		this.language = language;
	}



	public Map<String, Integer> getLexCatEntry() {
		return lexCatEntry;
	}



	public void setLexCatEntry(Map<String, Integer> lexCatentry) {
		this.lexCatEntry = lexCatentry;
	}



	public Map<String, String> getLexMetaLabel() {
		return lexWordLabels;
	}



	public void setLexMetaLabel(Map<String, String> lexMetaLabel) {
		this.lexWordLabels = lexMetaLabel;
	}


 
	public void setSaveOnChange(boolean saveOnChange) {
		this.saveOnChange = saveOnChange;
	}



	public void setWordFormsMappingFile(String wordFormsMappingFile) {
		this.wordFormsMappingFile = wordFormsMappingFile;
	}

	public void setPrintLevel(int printlevel) {
		printLevel = printlevel;
		out = new PrintLog(printLevel,true);
	}


	

	
	class FileDigester implements IndexedItemsCallbackIntf{
		 
		MultiDigester digester ;
		String[] files ;
		ArrayList<String>  filesources ;
		int closed=-1;
		public FileDigester(){
			
		}
		
		protected void digestingFiles(  ArrayList<String>  selectedSources, int threadcount){ 
			
			int i, n,id;
			String filename;
			int closed=-1;
			
			// providing also right now the callback address (=this class)
			// the interface contains just ONE routine: perform()
			digester = new MultiDigester(threadcount, (IndexedItemsCallbackIntf)this ) ;
			 
			// files = filesbypath;
			filesources = selectedSources ;
			
			// note, that the digester need not to know "anything" about our items, just the amount of items
			// we would like to work on.
			// the digester then creates simply an array of indices, which then point to the actual items,
			// which are treated anyway here (below) !
			digester.prepareItemSubSets( selectedSources.size(),0 );
			
			// digester.importDebugStrings( filesbypath ) ;
			
			if (out.getPrintlevel()<=2){
				digester.setDiagnosticPrintOut(0);
			}
			digester.setDiagnosticPrintOut(0); // 2 in case of problems
			
			
			// we could retrieve the array of sets of indices (ArrayList<Integer>) ...
			
			
			digester.execute() ;
			
			// the digester itself waits until all threads have been completed
			digester = null;
		}
		
		 
		// this will be called back, the mult-ithreaded digester selects an id, which is called from within one of the threads
		// the processID is just for fun... (and supervision)
		public void perform( int processID, int id ) {
			String src ;

			src = filesources.get(id);
			                                               out.print(5,"called back to <extracting> document, id ="+id+"  source ID = ... " ) ;
			
			
			// thats the call out to the parent class
			if ( src  != null){
				
				// accessDocument( files[id], processID ) ;
				
				// accessDocument( src, processID ) ;
				// loadSelectedFile(src, processID);
			}
			
		}

		@Override
		public int getClosedStatus() {
		
			return closed;
		}
 
	} // inner class FileDigester
	
}



